﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace UrlForm2
{
    public delegate void loadHandle(string obj);
    class Program
    {
       
        /// <summary>
        /// 应用程序的主入口点。
        /// </summary>
        [STAThread]
        static void Main()
        {
            Application.EnableVisualStyles();
            Application.SetCompatibleTextRenderingDefault(false);
            Application.Run(new Form1());
        }
    }
    class SimpleCrawler
    {
        public event loadHandle PageDownloaded;
        private Queue<string> preparedUrls = new Queue<string>();
        private System.Collections.Hashtable urls = new Hashtable();
        private int count = 0;
        public string startUrl;
      

        public void Crawl()
        {
            urls.Add(startUrl, false);//加入初始页面
            preparedUrls.Enqueue(startUrl);
            Console.WriteLine("开始爬行了.... ");
            while (true)
            {
                string current = null;
                for (int i = 0; i < preparedUrls.Count; i++)
                {
                    string url = preparedUrls.Dequeue();
                    if ((bool)urls[url]) continue;
                    else
                    {
                        current = url;
                        break;
                    }
                }

                if (current == null || count > 10) break;
                Console.WriteLine("爬行" + current + "页面!");
                PageDownloaded("爬行" + current + "页面!");
                string html = DownLoad(current); // 下载
                urls[current] = true;
                count++;
                
                Parse(html, current);//解析,并加入新的链接
               
                PageDownloaded("爬行结束");
            }
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                PageDownloaded(ex.Message);
                return "";
            }
        }

        private void Parse(string html, string current)
        {

            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                  .Trim('"', '\"', '#', '>');
              
                if (strRef.Length == 0 || !Regex.IsMatch(strRef, @"www.cnblogs.com")) continue;
                if (!Regex.IsMatch(strRef, "http"))//相对地址
                {
                    if (strRef[0] != '/')//直接拼接
                        strRef = current + strRef.Substring(1);
                }
                if (!preparedUrls.Contains(strRef)) preparedUrls.Enqueue(strRef);
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }
    }
}
